<?php

/**
 * 根据当当的产品分类列表采集产品列表
 * @param type $sqlfile
 * @param type $table
 * @param type $localCategoryId
 * @param type $arrbaseurl $baseurl=$arrbaseurl[$localCategoryId];
 * @param type $page
 * @param type $sort
 */
function Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page = 1, $sort = 'sort_saled_desc', $allpage = false) {

    if (!Task::$init && $allpage) {
        log::e('Please Init Task First!');
        die();
    }

    $baseurl = $arrbaseurl[$localCategoryId];
    $burl = $baseurl;
    $dataurl = $baseurl . '&page_index=' . $page . '&sort_type=' . $sort;

    if (!Task::isGoOn('localCategoryId', $localCategoryId, $dataurl) && $allpage) {
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId + 1, $arrbaseurl, $page, $sort, $allpage);
    }

    if (!Task::isGoOn('localCategoryId', $localCategoryId, $dataurl) && !Task::isGoOn('page', $page, $baseurl, true) && $allpage) {
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page + 1, $sort, $allpage);
    }

    $categoryid = $localCategoryId;
    $pList = '#javascript:AddToShoppingCart\((.*)\)#isU';
    $pQueHuo = '#javascript:AddPDNothing\((.*)\)#isU';


    $s = snoopy_fetch($dataurl);
    global $http_status_code;
    if ($http_status_code != 200) {
        log::w('Spider DangDang Http Status:' . $http_status_code, 'Now Sleep 10s And Try Again!');
        log::w($dataurl);
        sleep(10);
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page, $sort, $allpage);
    }
    if (strpos($s, iconv("UTF-8", "GB2312//IGNORE", '没有找到相关商品'))) {
        return false;
    }

    $pTotalCount = '#<span>' . iconv("UTF-8", "GB2312//IGNORE", '共') . '(.*)' . iconv("UTF-8", "GB2312//IGNORE", '页') . '#isU';
    preg_match_all($pList, $s, $d);
    preg_match_all($pTotalCount, $s, $totalCount);
    $totalPage = $totalCount[1][0];
    $pageProductIds = $d[1];

    if (!$totalPage || !is_numeric($totalPage)) {
        log::w('Spider DangDang TotalPage Error:' . $totalPage, 'Now Sleep 10s And Try Again!');
        log::w($dataurl);
        sleep(10);
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page, $sort, $allpage);
    }

    if ($page > $totalPage) {
        return false;
    }

    preg_match_all($pQueHuo, $s, $q);

    $urlArray = explode('=', $burl);
    $dangdangcategoryid = $urlArray[1];

    if (!is_array($pageProductIds) || empty($pageProductIds)) {
        log::w('Spider DangDang pageProductIds Error:' . print_r(pageProductIds, true), 'Now Sleep 10s And Try Again!');
        log::w($dataurl);
        sleep(10);
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page, $sort, $allpage);
    }

    foreach ($pageProductIds as $v) {
        $data = array();
        $data['dangdangproductid'] = $v;
        $data['categoryid'] = $categoryid;
        $data['dangdangcategoryid'] = $dangdangcategoryid;
        $data['page'] = $page;
        $data['fromurl'] = $dataurl;
        $query = makeInsertSQL($table, $data);
        if (!file_exists($sqlfile)) {
            FcheckDir($sqlfile);
        }
        file_put_contents($sqlfile, $query . PHP_EOL, 8);
    }

    if ($q[1]) {
        foreach ($q[1] as $v) {
            $data = array();
            $data['dangdangproductid'] = $v;
            $data['categoryid'] = $categoryid;
            $data['dangdangcategoryid'] = $dangdangcategoryid;
            $data['page'] = $page;
            $data['fromurl'] = $dataurl;
            $query = makeInsertSQL($table, $data);
            if (!file_exists($sqlfile)) {
                FcheckDir($sqlfile);
            }
            file_put_contents($sqlfile, $query . PHP_EOL, 8);
        }
    }

    if (Task::$init) {
        $array = array('localCategoryId' => $localCategoryId);
        Task::setCurrent($array);
        $array = array('page' => $page);
        Task::setCurrent($array);
    } else {
        if ($allpage) {
            log::e('Please Init Task First!');
            die();
        }
    }

    if ($page < $totalPage && $allpage) {
        return Spider_Dangdang_Get_Product_ListList($sqlfile, $table, $localCategoryId, $arrbaseurl, $page + 1, $sort, $allpage);
    }
}

function Spider_Dangdang_Get_Product_Book_Content($sqlfile, $intProductId, $intLocalCategoryId, $strDangdangCategoryId) {

    $sqldata = array();

	$sqldata['product_id'] = $intProductId;

    $testurl = "http://product.dangdang.com/product.aspx?product_id=" . $intProductId;
    $s = snoopy_fetch($testurl);

    $ptitle = '#<title>(.*)</title>#';
    preg_match_all($ptitle, $s, $titleArray);
    $title = $titleArray[1][0];
    $title = str_replace(' - 图书 - 当当网', '', $title);
    $title = str_replace(' - 当当网', '', $title);
	//_简介_书评_在线试读-当当图书
	$title = str_replace('_简介_书评_在线试读-当当图书', '', $title);
	$title = str_replace(iconv('UTF-8', 'GB2312//IGNORE', '_简介_书评_在线试读-当当图书'), '', $title);

    $title = str_replace(iconv('UTF-8', 'GB2312//IGNORE', ' - 图书 - 当当网'), '', $title);
    $title = str_replace(iconv('UTF-8', 'GB2312//IGNORE', ' - 当当网'), '', $title);
    $title = iconv('GB2312', 'UTF-8//IGNORE', $title);

    $sqldata['title'] = $title;
    $sqldata['intLocalCategoryId'] = $intLocalCategoryId;
    $sqldata['strDangdangCategoryId'] = $strDangdangCategoryId;


    $pkw = '#<meta name="keywords" content="(.*)" />#';
    preg_match_all($pkw, $s, $titleArray);
    $kw = $titleArray[1][0];
    $kw = iconv('GB2312', 'UTF-8//IGNORE', $kw);
    $pdescription = '#<meta name="keywords" content="(.*)" />#';
    preg_match_all($pdescription, $s, $titleArray);
    $description = $titleArray[1][0];
    $description = iconv('GB2312', 'UTF-8//IGNORE', $description);

    $sqldata['description'] = $description;
    $sqldata['kw'] = $kw;


    $pauthor = '#<li><span class="c1"><span class="ws2">' . iconv('UTF-8', 'GB2312//IGNORE', '作 者：') . '</span>(.*)</span></li>#isU';
    preg_match_all($pauthor, $s, $titleArray);
    $authorString = strip_tags($titleArray[1][0]);
    $authorString = iconv('GB2312', 'UTF-8//IGNORE', $authorString);
    $sqldata['authorString'] = $authorString;


    $ppubdate = '#<li><span class="c1">' . iconv('UTF-8', 'GB2312//IGNORE', '出版时间：') . '(.*)</span></li>#isU';
    preg_match_all($ppubdate, $s, $titleArray);
    $pubdate = addslashes(strip_tags($titleArray[1][0]));
    $pubdate = iconv('GB2312', 'UTF-8//IGNORE', $pubdate);
    $sqldata['pubdate'] = $pubdate;


    $pZiShu = '#<span class="c3"><span class="ws2">' . iconv('UTF-8', 'GB2312//IGNORE', '字 数：') . '</span>(.*)</span>#isU';
    preg_match_all($pZiShu, $s, $titleArray);
    $ZiShu = addslashes(strip_tags($titleArray[1][0]));
    $ZiShu = iconv('GB2312', 'UTF-8//IGNORE', $ZiShu);
    $sqldata['ZiShu'] = $ZiShu;


    $pZiShu = '#<span class="c3"><span class="ws2">' . iconv('UTF-8', 'GB2312//IGNORE', '页 数：') . '</span>(.*)</span>#isU';
    preg_match_all($pZiShu, $s, $titleArray);
    $ZiShu = addslashes(strip_tags($titleArray[1][0]));
    $ZiShu = iconv('GB2312', 'UTF-8//IGNORE', $ZiShu);
    $sqldata['YeShu'] = $ZiShu;


    $ppshe = '#<span class="ws1">' . iconv('UTF-8', 'GB2312//IGNORE', '出 版 社：') . '</span>(.*)</a></span></li>#isU';
    preg_match_all($ppshe, $s, $titleArray);
    $chubanshe = addslashes(strip_tags($titleArray[1][0]));
    $chubanshe = iconv('GB2312', 'UTF-8//IGNORE', $chubanshe);

    $sqldata['chubanshe'] = $chubanshe;

    $pdaohang = '#<div class="breadcrumb" name="__Breadcrumb_pub">(.*)</div>#isU';
    preg_match_all($pdaohang, $s, $daohangArray);

    $daohangString = strip_tags($daohangArray[1][0]);
    $daohangString = preg_replace('#\s+#', '', $daohangString);
    $daohangStringArray = explode('&gt;', $daohangString);

    $cid1 = $daohangStringArray[2];
    $cid2 = $daohangStringArray[3];
    $cid3 = $daohangStringArray[4];
    $cid4 = $daohangStringArray[5];

    $cid1 = iconv('GB2312', 'UTF-8//IGNORE', $cid1);
    $cid2 = iconv('GB2312', 'UTF-8//IGNORE', $cid2);
    $cid3 = iconv('GB2312', 'UTF-8//IGNORE', $cid3);
    $cid4 = iconv('GB2312', 'UTF-8//IGNORE', $cid4);

    $sqldata['cid1'] = trim(str_replace('&nbsp;','',$cid1));
    $sqldata['cid2'] = trim(str_replace('&nbsp;','',$cid2));
    $sqldata['cid3'] = trim(str_replace('&nbsp;','',$cid3));
    $sqldata['cid4'] = trim(str_replace('&nbsp;','',$cid4));

    $testarray = explode('=', $testurl);
    $url = 'http://product.dangdang.com/callback.php?type=detail&product_id=' . str_replace('&ref', '', $testarray[1]);
    $detailUrl = $url;
    $prdid = $testarray[1];
    $s = snoopy_fetch($detailUrl);

    $p = '#"content":(.*),"mediafeedback"#isU';

    preg_match_all($p, $s, $d);
    $detailBody = $d[1][0];

    $darray = json_decode('{"content":' . $detailBody . "}", true);

    if (is_array($darray) && $darray) {
        foreach ($darray as $k => $v) {
            $v = strip_tags($v['all']);
            $sqldata[$k] = $v;
        }
    }

    if (!file_exists($sqlfile)) {
        fCheckDir($sqlfile);
    }

	//print_r($sqldata);

    file_put_contents($sqlfile, makeInsertSQL('ddsql', $sqldata) . PHP_EOL, 8);
}

function Spider_Dangdang_Get_Product_Book_Comment($sqlfile, $intProductId) {
    $intDangdangId = $intProductId;
    if (!file_exists($sqlfile)) {
        fCheckDir($sqlfile);
    }
    $intMaxPage = mt_rand(2, 8);
    for ($pageIndex = 1; $pageIndex < $intMaxPage; $pageIndex++) {
        $url = 'http://product.dangdang.com/comment/main.php?product_id=' . $intProductId . '&page=' . $pageIndex . '&filtertype=1&type=part&iswidth=true';
        $s = snoopy_fetch($url);

        $pContent = '#<div class="s_cont">(.*)</div>#isU';
        $pTitle = '#<div class="title_top">(.*)</div>#isU';
        $pTitleLink = '#id="vote_helpful([\d]+)"#isU';
        $pDetail = '#<div class="comment_content">(.*)</div>#isU';

        $arrInsertData = array();

        preg_match_all($pContent, $s, $arrContent);
        preg_match_all($pTitle, $s, $arrTitle);
        preg_match_all($pTitleLink, $s, $arrTitleLinkId);

        if ($arrContent && $arrContent[1] && !empty($arrContent[1])) {
            foreach ($arrContent[1] as $k => &$data) {
                $data = strip_tags($data);
                $data = str_replace(iconv('UTF-8', 'GB2312//IGNORE', '<!-- 评论内容 -->'), '', $data);
                $data = preg_replace('#\s+#', '', $data);
                $data = str_replace(iconv('UTF-8', 'GB2312//IGNORE', '<!--评论内容-->'), '', $data);
                $arrInsertData[$k]['content'] = iconv('GB2312', 'UTF-8//IGNORE', $data);
            }unset($data);
        }

        if ($arrTitle && $arrTitle[1] && !empty($arrTitle[1])) {
            foreach ($arrTitle[1] as $k => &$data) {
                $data = strip_tags($data);
                $data = str_replace(iconv('UTF-8', 'GB2312//IGNORE', '<!-- 评论标题 -->'), '', $data);
                $data = preg_replace('#\s+#', '', $data);
                $data = str_replace(iconv('UTF-8', 'GB2312//IGNORE', '<!--评论标题-->'), '', $data);
                $arrInsertData[$k]['title'] = iconv('GB2312', 'UTF-8//IGNORE', $data);
            }unset($data);
        }

        if ($arrTitleLinkId && $arrTitleLinkId[1] && !empty($arrTitleLinkId[1])) {
            foreach ($arrTitleLinkId[1] as $k => &$data) {
                $data = preg_replace('#\s+#', '', $data);
                $arrInsertData[$k]['commentId'] = $data;
            }unset($data);
        }

        if (!empty($arrInsertData)) {

            foreach ($arrInsertData as $k => &$data) {
                if (strpos($data['content'], iconv('UTF-8', 'GB2312//IGNORE', '更多') . '&gt;&gt;')) {
                    $commentDetailUrl = 'http://comm.dangdang.com/member/myreviewdetail.php?review_id=' . $data['commentId'];
                    $sDetail = snoopy_fetch($commentDetailUrl);
                    preg_match_all($pDetail, $sDetail, $arrCommentDetail);
                    if ($arrCommentDetail[1][0]) {
                        $arrCommentDetail[1][0] = strip_tags($arrCommentDetail[1][0]);
                        $arrCommentDetail[1][0] = preg_replace('#\s+#', '', $arrCommentDetail[1][0]);
                        $data['content'] = iconv('GB2312', 'UTF-8//IGNORE', $arrCommentDetail[1][0]);
                    }
                }
                $data['product_id'] = $intDangdangId;
                $query = makeInsertSQL('ddcomment', $data);

                file_put_contents($sqlfile, $query . PHP_EOL, 8);
            }unset($data);
        } else {
            $pageIndex = $intMaxPage;
        }
    }
}

function Spider_Dangdang_Get_Product_Book_Cover($coverdir, $intProductId, $categoryid) {

    $bookid = $intProductId;

    $bookfile = $coverdir . '/' . $categoryid . '/' . substr(md5($bookid), 0, 2) . '/' . $bookid . '.jpg';
    fCheckDir($bookfile);
    $dx = substr($bookid, -1);
    $testurl = 'http://img3' . $dx . '.ddimg.cn/' . ($bookid % 99) . '/' . ($bookid % 37) . '/' . $bookid . '-1_b.jpg';

    $s = snoopy_fetch($testurl);

    file_put_contents($bookfile, $s);

    if (filesize($bookfile) == 4048 || !filesize($bookfile)) {
        file_put_contents($coverdir . '/4048.txt', $bookfile . PHP_EOL, 8);
        unlink($bookfile);
        $query = "UPDATE ddsql SET haspic='0' WHERE `product_id`='{$intProductId}' LIMIT 1;";
        file_put_contents($coverdir . '/4048.sql', $query . PHP_EOL, 8);
    }
}


function Spider_Dangdang_Get_Product_Book_Cover_URL($coverdir, $intProductId, $categoryid) {

    $bookid = $intProductId;

    $bookfile = $coverdir . '/' . $categoryid . '/' . substr(md5($bookid), 0, 2) . '/' . substr(md5($bookid), 4, 2) . '/' . $bookid . '.jpg';
    $dx = substr($bookid, -1);
    $testurl = 'http://img3' . $dx . '.ddimg.cn/' . ($bookid % 99) . '/' . ($bookid % 37) . '/' . $bookid . '-1_b.jpg';
	return $testurl;
}

?>